/* The ns package provides the net-score.org functionality.  */
package ns

/*Indexing functions.

These methods are intended to be used on backend instances to process larger
amounts of data.  Often there is a method that requires no parameters that can
be easily included in a cron file that calls methods with calculated parameters.
*/

import (
	"appengine"
	"appengine/datastore"
	"appengine/log"
	"appengine/taskqueue"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

func init() {
	http.HandleFunc("/_ah/start", handleStart)
	http.HandleFunc("/_ah/stop", handleStop)
	http.HandleFunc("/ns/updateIndex", updateIndex)
	http.HandleFunc("/ns/updateRecords", updateRecords)
	http.HandleFunc("/ns/recordsWorker", recordsWorker)
}

/*
Warning do not change order of these!
Append only and update the String method.
*/
type TestFormat int

const (
	Flock = iota
	Fowl
	Feather
	Ostrich
	Egg
	Nsdt
)

var (
	toleranceValue       = int64(10)
	ErrUnknownTestFormat = errors.New("unknown test format")
	ErrBadParameters     = errors.New("bad parameters")
)

//TODO(gavaletz) This should return and error in Unknown case.
func (tf TestFormat) String() string {
	switch tf {
	case Flock:
		return "Flock"
	case Fowl:
		return "Fowl"
	case Feather:
		return "Feather"
	case Ostrich:
		return "Ostrich"
	case Egg:
		return "Egg"
	case Nsdt:
		return "Nsdt"
	}
	return "Unknown"
}

func GetTestFormat(s string) (TestFormat, error) {
	switch s {
	case "Flock":
		return Flock, nil
	case "Fowl":
		return Fowl, nil
	case "Feather":
		return Feather, nil
	case "Ostrich":
		return Ostrich, nil
	case "Egg":
		return Egg, nil
	case "Nsdt":
		return Nsdt, nil
	}
	return Flock, ErrUnknownTestFormat
}

/*
Warning do not change order of these!
Append only and update the String method.
*/
type ConnectionType int

const (
	WiredLAN = iota
	WiredCable
	WiredDSL
	WifiLAN
	WifiCable
	WifiDSL
	Mobile3G
	Mobile4G
	Other
)

func (ct ConnectionType) String() string {
	switch ct {
	case WiredLAN:
		return "Wired LAN"
	case WiredCable:
		return "Wired Cable"
	case WiredDSL:
		return "Wired DSL"
	case WifiLAN:
		return "Wifi LAN"
	case WifiCable:
		return "Wifi Cable"
	case WifiDSL:
		return "Wifi DSL"
	case Mobile3G:
		return "Mobile 3G"
	case Mobile4G:
		return "Mobile 4G"
	case Other:
		return "Other"
	}
	return "Unknown"
}

func handleStart(w http.ResponseWriter, r *http.Request) {
	// This handler is executed when a backend instance is started.
	// If it responds with a HTTP 2xx or 404 response then it is ready to 
	// go.  Otherwise, the instance is terminated and restarted.
	// The instance will receive traffic after this handler returns.

	c := appengine.NewContext(r)
	http.HandleFunc("/ns/updateIndexWorker/", updateIndexWorker)
	c.Infof("indexer backend started.")
	fmt.Fprint(w, "OK")
}

func handleStop(w http.ResponseWriter, r *http.Request) {
	// This handler is executed when a backend instance is being shut down.
	// It has 30s before it will be terminated.
	// When this is called, no new requests will reach the instance.

	c := appengine.NewContext(r)
	//Should we unregister the worker when the backend is down?
	//http.DefaultServeMux.m["/ns/updateIndexWorker/"].DELETE
	c.Infof("indexer backend stopped.")
}

//TODO(gavaletz) Method to return a given time range.

//TODO(gavaletz) #41 Index data
/*
For the short term we should work on 10 or so at a time
*/
func updateIndexWorker(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	name, index := appengine.BackendInstance(c)
	hostname := appengine.BackendHostname(c, name, index)
	c.Debugf("appengine.IsDevAppServer = %v", appengine.IsDevAppServer())
	c.Debugf("appengine.DefaultVersionHostname = %v",
		appengine.DefaultVersionHostname(c))
	c.Debugf("appengine.VersionID = %v", appengine.VersionID(c))
	c.Debugf("appengine.AppID = %v", appengine.AppID(c))
	c.Debugf("appengine.InstanceID = %v", appengine.InstanceID())
	c.Debugf("appengine.Datacenter = %v", appengine.Datacenter())
	c.Debugf("appengine.BackendInstance.name = %v", name)
	c.Debugf("appengine.BackendInstance.index = %v", index)
	c.Debugf("appengine.BackendHostname = %v", hostname)

	//grab un-indexed fields or those missing the index tag
	//add default values for missing values
	//unpack Header
	//unpack JSON grab what we need and then zip up the remaining

	q := datastore.NewQuery("RawData").Filter("Indexed =", false)
	var n int
	var err error
	if n, err = q.Count(c); err != nil {
		c.Errorf("RESULTS q.Count for %s err = %v", r.RemoteAddr, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	tds := make([]RawData, n)
	if _, err := q.GetAll(c, &tds); err != nil {
		c.Errorf("RESULTS q.GetAll for %s err = %v", r.RemoteAddr, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, "updateIndexWorker OK")
}

func updateIndex(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	backEndHost := appengine.BackendHostname(c, "indexer", -1)
	c.Debugf("appengine.BackendHostname = %v", backEndHost)
	name := fmt.Sprintf("testIndex-%d", time.Now().Unix())
	t := &taskqueue.Task{
		Path:   "/ns/updateIndexWorker/",
		Name:   name,
		Header: make(http.Header),
		Method: "GET",
	}
	t.Header.Set("Host", backEndHost)
	t.Header.Set("Content-Type", "message/http")

	if _, err := taskqueue.Add(c, t, "indexer"); err != nil {
		c.Errorf("TASKQUEUE err = %v", err)
		return
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, "updateIndex OK")
}

/*This method provides a hook for cron jobs to kick off recordsWorker requests.

This method can be run at any time of the day, and it will calculate the begin
and end of the previous day and issue a call to the recordsWorker method with
these start and end times.  The call is placed in a taskqueue so that it can be
handled at an appropriate time.

HTTP parameters:
	None
*/
func updateRecords(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	now := time.Now()
	backEndHost := appengine.BackendHostname(c, "indexer", -1)
	c.Infof("updateRecords:appengine.BackendHostname = %v", backEndHost)
	name := fmt.Sprintf("testIndex-%d", now.Unix())

	// Calculate the start and end times based on the previous day.
	delta, err := time.ParseDuration("24h")
	if err != nil {
		c.Errorf("updateRecords:time.ParseDuration err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	d0 := now.Add(-delta)
	startTime := time.Date(d0.Year(), d0.Month(), d0.Day(), 0, 0, 0, 0,
		d0.Location())
	endTime := startTime.Add(delta)

	v := url.Values{}
	v.Set("s", strconv.FormatInt(startTime.Unix(), 10))
	v.Set("e", strconv.FormatInt(endTime.Unix(), 10))
	v.Set("v", strings.Split(appengine.VersionID(c), ".")[0])

	p := url.URL{
		Path:     "/ns/recordsWorker",
		RawQuery: v.Encode(),
	}

	t := &taskqueue.Task{
		Path:   p.String(),
		Name:   name,
		Header: make(http.Header),
		Method: "GET",
	}
	t.Header.Set("Host", backEndHost)
	t.Header.Set("Content-Type", "message/http")

	if _, err := taskqueue.Add(c, t, "indexer"); err != nil {
		c.Errorf("TASKQUEUE err = %v", err)
		return
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, "updateRecords OK")
}

/*This method attaches log Records to the TestRecord and updates site counts.

It is VERY IMPORTANT THAT THIS IS NOT RUN MORE THAN ONCE FOR THE SAME TIME RANGE
the implications are that it will count the test records twice for a given site.

Since every test has a GET request to get the JavaScript code and a POST request
to store the results we can map these log entries to the tests that generated
them.  With this in place we no longer need to download the log files to a
separate system (although that is still not a bad idea in case something goes
wrong at a deeper level with GAE).

At this point there is a fair amount of overlap in the data that is stored in a
TestRecord and the data that is stored in the log Records.

HTTP parameters:
	/ns/recordsWorker?s=<startTime>&e=<endTime>&v=<versionID>

	startTime is the earliest TestRecord or log Record that will be
	considered, and the endTime is the latest.  We extend these times by 
	toleranceValue seconds on either side to make sure that we capture any
	boundry cases.  The versionID is the major version ID for the logs you
	want to target.
*/
func recordsWorker(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	var start int64
	var end int64
	var err error
	start, err = strconv.ParseInt(r.FormValue("s"), 0, 64)
	if err != nil {
		c.Errorf("recordsWorker:strconv.ParseInt err = %v", err)
		http.Error(w, err.Error(),
			http.StatusInternalServerError)
		return
	}
	end, err = strconv.ParseInt(r.FormValue("e"), 0, 64)
	if err != nil {
		c.Errorf("recordsWorker:strconv.ParseInt err = %v", err)
		http.Error(w, err.Error(),
			http.StatusInternalServerError)
		return
	}
	version := r.FormValue("v")
	if version == "" || start >= end {
		err := ErrBadParameters
		c.Errorf("recordsWorker: err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Setup the queries.
	qLogRecords := &log.Query{
		Incomplete:    false,
		AppLogs:       false,
		ApplyMinLevel: false,
		Versions:      []string{version},
	}
	qTestRecords := datastore.NewQuery("TestRecord").Order("-GetTime")
	qSiteData := datastore.NewQuery("SiteData")

	// Limit the queries based request on parameters.
	if start > 0 {
		qLogRecords.StartTime = time.Unix(start-toleranceValue, 0)
		qTestRecords = qTestRecords.Filter("GetTime >", start)
	}
	if end > 0 {
		qLogRecords.EndTime = time.Unix(end+toleranceValue, 0)
		qTestRecords = qTestRecords.Filter("GetTime <", end)
	}

	// Load the Records into a LogList.
	logList := &LogList{}
	results := qLogRecords.Run(c)
	for {
		record, err := results.Next()
		if err == log.Done {
			break
		}
		if err != nil {
			c.Errorf("recordsWorker:results.Next err = %v", err)
			break
		}

		//TODO(gavaletz) We should count and do something with these.
		// Filter out records that we are not interested in to reduce
		// the overhead when matching.
		if record.Status != 200 {
			continue
		}
		if strings.HasSuffix(record.Resource, ".js") {
			continue
		}
		if strings.HasSuffix(record.Resource, ".css") {
			continue
		}
		if strings.HasSuffix(record.Resource, ".ico") {
			continue
		}
		if strings.HasSuffix(record.Resource, ".xml") {
			continue
		}
		if strings.HasSuffix(record.Resource, ".txt") {
			continue
		}
		if strings.HasPrefix(record.Resource, "/ns") {
			continue
		}
		if strings.HasPrefix(record.Resource, "/user") {
			continue
		}
		if strings.HasPrefix(record.Resource, "/_ah") {
			continue
		}
		if strings.HasPrefix(record.Resource, "/videoID") {
			continue
		}
		err = logList.Push(record.Method, record.Resource, record.IP,
			record.StartTime.Unix(), record)
		// This error is quite common and can be cause by browsers like
		// Chrome making lots of requests.  It is safe to ignore it.
		if err == LogListCollisionError {
			continue
		}
		// Most likely a LogListUnknownError, and indicates a problem in
		// the logic of the LogList.
		if err != nil {
			c.Errorf("recordsWorker:logList.Push err = %v", err)
			break
		}
	}
	if logList.Size() <= 0 {
		c.Warningf("recordsWorker:logList.Size = %d", logList.Size())
		http.NotFound(w, r)
		//return
	}
	c.Infof("logList.Size() = %d", logList.Size())
	logList.SetTolerance(toleranceValue)

	// Get the SiteData based on the query and map to DevKey.
	var sds []SiteData
	sdks, err := qSiteData.GetAll(c, &sds)
	if err != nil {
		c.Errorf("recordsWorker:q.GetAll err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	sdm := make(map[string]*SiteData, len(sds))
	for i := range sds {
		sdm[sds[i].DevKey] = &sds[i]
	}

	// Get the TestRecords based on the query.
	var trs []TestRecord
	trks, err := qTestRecords.GetAll(c, &trs)
	if err != nil {
		c.Errorf("recordsWorker:q.GetAll err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Match the Records with the Test Records
	for i := range trs {
		var resource string
		var sd *SiteData
		switch trs[i].TestForm {
		case Flock:
			resource = "/flock"
		case Fowl:
			resource = fmt.Sprintf("/%s/", "fowl")
		case Feather:
			resource = fmt.Sprintf("/%s/", "feather")
		case Ostrich:
			resource = fmt.Sprintf("/%s/", "ostrich")
		case Egg:
			resource = fmt.Sprintf("/%s/", "egg")
		case Nsdt:
			resource = fmt.Sprintf("/%s/", "nsdt")
		default:
			c.Errorf("recordsWorker: err = %v",
				ErrUnknownTestFormat)
			continue
		}
		// Unless we have "ns" as the dev key add it to the resource.
		if trs[i].DevKey != "ns" {
			resource = fmt.Sprintf("%s%s", resource, trs[i].DevKey)
			sd = sdm[trs[i].DevKey]
			if sd != nil {
				sd.RequestCount += 1
			}
		}
		record, err := logList.Match("GET", resource, trs[i].RemoteAddr,
			trs[i].GetTime)
		if err == LogListMatchNotFoundError {
			c.Infof("recordsWorker:logList.Match err = %v", err)
		} else if err != nil {
			c.Errorf("recordsWorker:logList.Match err = %v", err)
		} else {
			ks := fmt.Sprintf("%d_%s_GET", trs[i].GetTime,
				trs[i].RemoteAddr)
			trs[i].GetRecordKey = datastore.NewKey(c, "Record", ks,
				0, nil)
			_, err = datastore.Put(c, trs[i].GetRecordKey, record)
			if err != nil {
				c.Errorf("recordsWorker:datastore.Put err = %v",
					err)
			}
		}
		// This indicates data and we should have a POST record.
		if trs[i].TestDataKey != nil {
			if sd != nil {
				sd.TestCount += 1
			}
			record, err = logList.Match("POST", "/upload/",
				trs[i].RemoteAddr, trs[i].PostTime)
			if err == LogListMatchNotFoundError {
				c.Infof("recordsWorker:logList.Match err = %v",
					err)
				continue
			}
			if err != nil {
				c.Errorf("recordsWorker:logList.Match err = %v",
					err)
				continue
			}
			ks := fmt.Sprintf("%d_%s_POST", trs[i].GetTime,
				trs[i].RemoteAddr)
			trs[i].PostRecordKey = datastore.NewKey(c, "Record", ks,
				0, nil)
			_, err = datastore.Put(c, trs[i].PostRecordKey, record)
			if err != nil {
				c.Errorf("recordsWorker:datastore.Put err = %v",
					err)
			}
		}
	}

	// Write the TestRecords to the datastore.
	_, err = datastore.PutMulti(c, trks, trs)
	if err != nil {
		c.Errorf("recordsWorker:datastore.PutMulti err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	// Write the SiteData to the datastore.
	_, err = datastore.PutMulti(c, sdks, sds)
	if err != nil {
		c.Errorf("recordsWorker:datastore.PutMulti err = %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, "recordsWorker OK")
}
